package cn.edu.hitsz.spider;

import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import org.apache.log4j.Logger;

import populus.core.queue.URLLinkedBlockingQueue;
import populus.core.queue.UrlItem;
import cn.edu.hitsz.backup.UncommitedQueue;
import cn.edu.hitsz.conf.SpiderConfig;
import cn.edu.hitsz.http.HTTPRequest;
import cn.edu.hitsz.task.FetchTask;

/**
 * A SIMPLE SPIDER TO FECTH THE PAGE
 */
public class SimpleSpider implements Runnable {
	private final static Logger log = Logger.getLogger(SpiderConfig.class);


	private SpiderConfig config = null;
	private ThreadPoolExecutor tpe =null;

	/**
	 * 
	 * Constructor function of Class
	 */
	public SimpleSpider() {
		init();
	}

	public void run() {

		while(!URLLinkedBlockingQueue.linkqueue.isEmpty()){
			try {
				// Take UrlItem from URLqueue and offer to uncommited queue
				UrlItem ui = URLLinkedBlockingQueue.linkqueue.take();
				UncommitedQueue.uncommited.offer(ui);

				// send request and revoke connection
				HTTPRequest req = new HTTPRequest(ui);
				tpe.execute(new FetchTask(req,config.getDataconn()));

				log.info("coreSize:"+tpe.getPoolSize()+" waits:"+tpe.getQueue().size());
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		}
	}

	/**
	 * Init the property file and set the parameters necessary
	 * 
	 * void 2015年8月13日
	 */
	public void init() {
		// Load the property files conf.ini
		try {
			Properties pps = new Properties();
			pps.load(new InputStreamReader(new FileInputStream("conf.ini")));
			config = new SpiderConfig(pps);
		} catch (Exception e) {
			e.printStackTrace();
		}
		//set the thread pool
		tpe = new ThreadPoolExecutor(
				config.getSpiderThreads(),10,200,
				TimeUnit.MILLISECONDS,new ArrayBlockingQueue<Runnable>(5),
				new ThreadPoolExecutor.DiscardPolicy()
				);
		// push the starts urls to the queue
		pushStartUrls();

	}

	/**
	 * Push the start urls into the blockingqueue
	 * 
	 * void 2015年8月13日
	 */
	public void pushStartUrls() {
		Set<String> starts = config.getStartURLs();
		for (String url : starts) {
			UrlItem ui = new UrlItem(url, 0);
			try {
				URLLinkedBlockingQueue.linkqueue.put(ui);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
		}
	}
}
